#Read and mean impute the training data
setwd("~/accept_decline")

mean_impute = function(x)
{
	mu = mean(na.omit(x))
	x[is.na(x)] = mu
	x
}

df = readRDS("train.rds")
df = data.frame(lapply(df, mean_impute))
df = data.frame(df, loss_ind=df$loss>0)

#Compute the Tantrev variable
m = glm(formula = loss_ind ~ f274 + f528, family = binomial(), data = df)
pred_tantrev = predict(m, type="response")
a = cut(pred_tantrev, quantile(pred_tantrev, seq(0, 1, .01), include.lowest=T))
plot(tapply(df$loss_ind, a, mean))
sum(df$loss_ind[pred_tantrev>.09])/sum(df$loss_ind)

quantilize = function(x)
{
  qntls = unique(quantile(x, seq(0, 1, .05)))
  print(length(qntls))
  if(length(qntls)>1)cut(x, qntls, 1:(length(qntls) - 1), include.lowest=T)
  else as.factor(rep(1, length(x)))
}

df2 = df[pred_tantrev > .09,]
pred_tantrev2 = pred_tantrev[pred_tantrev > .09]
a = cut(pred_tantrev2, quantile(pred_tantrev2, seq(0, 1, .2), include.lowest=T))
df_quantile = lapply(df2[,2:(ncol(df2) - 2)], quantilize)
df_quantile = data.frame(df_quantile, tantrev = a, loss_ind=df2$loss_ind, loss=df2$loss)
m_fine = glm(loss_ind ~ tantrev + f13 + f67 + f84 + f404 + f514 + f527 + f201 + f211 + f238,
             data=df_quantile, family=binomial)

pred = predict(m_fine, type="response", newdata=df_quantile)
pred[is.na(pred)] = 0
plot(quantile(pred, seq(0, 1, .05)), type="l")
b=cut(pred, quantile(pred, seq(0, 1, .05), include.lowest=T))
points(tapply(df_quantile$loss_ind, b, mean))
tapply(df_quantile$loss, b, median)


##Now need to apply to test, which means test variables must be quantilized
quantilize2 = function(x,y)
{
	qntls = unique(quantile(x, seq(0, 1, .05)))
	print(length(qntls))
	if(length(qntls)>1)cut(y, qntls, 1:(length(qntls) - 1), include.lowest=T)
	else as.factor(rep(1, length(y)))
}

df_test = readRDS("test.rds")
df_test = data.frame(id=df_test$id, lapply(
  df_test[ ,c("f13", "f67", "f84", "f404", "f514", "f527", "f201", "f211", 
              "f238", "f274", "f528")],	mean_impute))
pred_test_tantrev = predict(m, newdata=df_test, type="response")
df_test2 = df_test[pred_test_tantrev>.09,]
pred_test_tantrev2 = pred_test_tantrev[pred_test_tantrev>.09]
b = cut(pred_test_tantrev2, quantile(pred_tantrev2, seq(0, 1, .2), include.lowest=T))

df_test_quantile = data.frame(id=df_test2$id, 
f13=quantilize2(df2$f13, df_test2$f13),
f67=quantilize2(df2$f67, df_test2$f67),
f84=quantilize2(df2$f84, df_test2$f84),
f404=quantilize2(df2$f404, df_test2$f404),
f514=quantilize2(df2$f514, df_test2$f514),
f527=quantilize2(df2$f527, df_test2$f527),
f201=quantilize2(df2$f201, df_test2$f201),
f211=quantilize2(df2$f211, df_test2$f211),
f238=quantilize2(df2$f238, df_test2$f238),
tantrev = b)

pred_test = predict(m_fine, type="response", newdata=df_test_quantile)

test_loss = rep(0, nrow(df_test2))

 (0.346,0.412]  (0.412,0.478]  (0.478,0.536]  (0.536,0.587]  (0.587,0.635] 
             0              0              0              1              1 
 (0.635,0.677]  (0.677,0.712]  (0.712,0.744]  (0.744,0.774]  (0.774,0.802] 
             2              3              3              3              4 
 (0.802,0.828]  (0.828,0.854]  (0.854,0.881]  (0.881,0.911]  (0.911,0.985] 
             4              5              6              6              8 


test_loss[pred_test>.536 & pred_test<=.635] = 1
test_loss[pred_test>.635 & pred_test<=.677] =2
test_loss[pred_test>.677 & pred_test<=.774] = 3
test_loss[pred_test>.774 & pred_test<=.828] = 4
test_loss[pred_test>.828 &pred_test<=.854] = 5
test_loss[pred_test>.854 &pred_test<=.911] = 6
test_loss[pred_test>.911] = 8


df_ans = data.frame(id=df_test2$id, test_loss)
df_ans2 = merge(data.frame(id=df_test$id), df_ans, by="id", all.x=T)
names(df_ans2)=c("id","loss")
df_ans2$loss[is.na(df_ans2$loss)] = 0

write.csv(df_ans2,"submission.csv", row.names=F, quote=F)
